{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用户和活动关联关系处理\n",
    "\n",
    "\n",
    "整个数据集中活动数目(events.csv)太多, 所以下面的处理中, 我们找出只在训练集和测试集中出现的活动和用户集合, 并对他们重新编制索引"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 保存数据\n",
    "import pickle\n",
    "# 迭代器\n",
    "import itertools\n",
    "# 处理时间字符串\n",
    "import datetime\n",
    "\n",
    "import numpy as np\n",
    "import scipy.io as sio # 文件 IO\n",
    "import scipy.sparse as ss # 稀疏矩阵库\n",
    "\n",
    "# 相似度/距离\n",
    "import scipy.spatial.distance as ssd\n",
    "\n",
    "from collections import defaultdict # 带默认值的字典\n",
    "from sklearn.preprocessing import normalize # 标准化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of uniqueUsers: 3391\n",
      "Number of uniqueEvents: 13418\n"
     ]
    }
   ],
   "source": [
    " \"\"\"\n",
    "我们只关心 train 和 test 中出现的 user 和 event, 因此重点处理这部分关联数据\n",
    "\n",
    "train.csv 有 6 列:\n",
    "user: 用户 ID\n",
    "event: 活动 ID\n",
    "invited: 是否被邀请(0/1)\n",
    "timestamp: ISO-8601 UTC 格式时间字符串, 表示用户看到该活动的时间\n",
    "interested: 是否感兴趣(0/1)\n",
    "not_interested: 是否不感兴趣(0/1)\n",
    "\n",
    "Test.csv 除了没有interested 和 not_interested, 其余列与 train 相同\n",
    " \"\"\"\n",
    "# 统计训练集中有多少不的 users 和 events\n",
    "uniqueUsers = set()\n",
    "uniqueEvents = set()\n",
    "\n",
    "# 倒排表\n",
    "# 统计每个用户参加的活动和每个活动参加的用户\n",
    "eventsForUser = defaultdict(set)\n",
    "usersForEvent = defaultdict(set)\n",
    "\n",
    "for filename in ['train.csv', 'test.csv']:\n",
    "    f = open(file=filename, mode='rb') # 以 binary 形式读\n",
    "    # 忽略第一行(列名字), strip()去除首尾空格等字符\n",
    "    f.readline().strip().split(','.encode(encoding='utf-8'))\n",
    "    \n",
    "    for line in f: # 对每条记录\n",
    "        cols = line.strip().split(','.encode(encoding='utf-8')) # 读每条记录\n",
    "        uniqueUsers.add(cols[0]) # 第 1 列为用户 ID\n",
    "        uniqueEvents.add(cols[1]) # 第 2 列为活动 ID\n",
    "        \n",
    "    f.close() # 关闭文件\n",
    "    \n",
    "n_uniqueUsers = len(uniqueUsers) # 唯一用户数\n",
    "n_uniqueEvents = len(uniqueEvents) # 唯一活动数\n",
    "\n",
    "print('Number of uniqueUsers: %d' %n_uniqueUsers)\n",
    "print('Number of uniqueEvents: %d' %n_uniqueEvents)\n",
    "\n",
    "# 用户关系矩阵, 可用于后续 LFM/SVD++ 处理的输入\n",
    "# 这是一个稀疏矩阵, 记录用户对活动感兴趣\n",
    "userEventScores = ss.dok_matrix((n_uniqueUsers, n_uniqueEvents))\n",
    "userIndex = dict()\n",
    "eventIndex = dict()\n",
    "\n",
    "# 重新编码用户索引字典\n",
    "for i, u in enumerate(uniqueUsers):\n",
    "    userIndex[u] = i\n",
    "    \n",
    "# 重新编码活动索引字典\n",
    "for i, e in enumerate(uniqueEvents):\n",
    "    eventIndex[e] = i\n",
    "    \n",
    "n_records = 0\n",
    "ftrain = open('train.csv', 'rb')\n",
    "ftrain.readline()\n",
    "for line in ftrain:\n",
    "    cols = line.strip().split(','.encode(encoding='utf-8'))\n",
    "    i = userIndex[cols[0]] # 用户新索引\n",
    "    j = eventIndex[cols[1]] # 活动新索引\n",
    "    \n",
    "    eventsForUser[i].add(j) # 用户参加的活动添加\n",
    "    usersForEvent[j].add(i) # 参加该活动的用户添加\n",
    "    \n",
    "    score = int(cols[4]) # interested\n",
    "    \n",
    "    userEventScores[i, j] = score\n",
    "    \n",
    "ftrain.close()\n",
    "\n",
    "# 统计每个用户参加的活动并保存, 后续用于将用户朋友参加的活动影响到用户\n",
    "pickle.dump(obj=eventsForUser, file=open('PE_eventsForUser.pkl', 'wb'))\n",
    "# 统计每个活动有哪些用户参加并保存\n",
    "pickle.dump(obj=usersForEvent, file=open('PE_usersForEvent.pkl', 'wb'))\n",
    "\n",
    "# 保存用户-活动关系矩阵 R, 以备后用\n",
    "sio.mmwrite(target='PE_userEventScores', a=userEventScores)\n",
    "\n",
    "# 保存用户索引表\n",
    "pickle.dump(userIndex, open('PE_userIndex.pkl', 'wb'))\n",
    "# 保存活动索引表\n",
    "pickle.dump(eventIndex, open('PE_eventIndex.pkl', 'wb'))\n",
    "\n",
    "# 为了防止不必要的计算, 我们找出所有关联的 user 或者关联的 event\n",
    "# 所谓的关联 user, 指的是至少在同一个 event 上有行为的 user pair\n",
    "# 所谓的关联 event, 指的是至少有同一个 user 有行为的 event pair\n",
    "uniqueUserPairs = set()\n",
    "uniqueEventPairs = set()\n",
    "for event in uniqueEvents:\n",
    "    i = eventIndex[event]\n",
    "    users = usersForEvent[i]\n",
    "    if len(users) > 2: # 参加该活动用户超过 2 人\n",
    "        uniqueUserPairs.update(itertools.combinations(users, 2)) # 生成用户对(无重复)\n",
    "        \n",
    "for user in uniqueUsers:\n",
    "    u = userIndex[user]\n",
    "    events = eventsForUser[u]\n",
    "    if len(events) > 2:\n",
    "        uniqueEventPairs.update(itertools.combinations(events, 2))\n",
    "\n",
    "# 保存用户-活动关系对索引表\n",
    "pickle.dump(uniqueUserPairs, open('FE_uniqueUserPairs.pkl', 'wb'))\n",
    "pickle.dump(uniqueEventPairs, open('PE_uniqueEventPairs.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练集和测试集中出现的用户数目和活动数目远小于 users.csv 出现的用户数和 events.csv 出现的活动数"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
